#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :run_movie_reviews.py
# @Time      :2023/10/12 
# @Author    :CL
# @email     :1037654919@qq.com
from multiprocessing import Pool
from bs4 import BeautifulSoup
from lxml import html

from douban import douban
from utils import mongo_manager



def save_movie_reviews(seed):
    movie_id = seed['_id']
    douban_movie_reviews = mongo_manager('douban_movie_reviews')
    douban_movie_detail = mongo_manager('douban_movie_detail', db='douban')
    url = f'https://movie.douban.com/subject/{movie_id}/reviews'
    # urlr2 = 'https://movie.douban.com/subject/1291561/reviews?start=20'
    response = douban.get_movie_reviews(url)
    # print(response['html'])
    soups = BeautifulSoup(response['html'], 'lxml')
    # elements =html.fromstring(response['html']).xpath('//div[@class="main review-item"]')
    # # Print the elements
    # for element in elements:
    #     print(html.tostring(element, pretty_print=True, encoding='unicode'))
    #     data = BeautifulSoup(html.tostring(element, pretty_print=True, encoding='unicode'),'lxml')
    datas  = soups.find_all('div',class_='main review-item')
    for data in datas: #todo
        print(data)
        username = data.find('header').find('a',class_ ='name').text.strip()
        times = data.find('header').find('span',class_ ='main-meta').text.strip()
        try:
            title = data.find('h2'
                              '').finf('a').text.strip()
            href = data.find('h2').finf('a')['href']
            _id = str(href).split('/')[-2]
            short_content = data.find('div',class_='short-content').text.strip()
            action_agree = data.find('div',class_='action').find('a',title = '有用').text.strip()
            action_disagree = data.find('div',class_='action').find('a',title = '没用').text.strip()
            comments = data.find('div',class_='action').find('a',class_ = 'reply ').text.strip()
        except:
            title =''
            href =''
            _id = ''
            short_content =''
            action_agree =''
            action_disagree =''
            comments= ''

        insert_data = {'_id': _id,'movie_id':movie_id, 'username': username, 'times': times, 'title': title, 'href': href, 'short_content': short_content,
                       'action_agree': action_agree, 'action_disagree': action_disagree,
                       'comments': comments
                       }
        print(insert_data)
        try:
            # print(data)
            douban_movie_reviews.insertOne(insert_data)
        except Exception as e:
            douban_movie_reviews.updateOne({'_id': insert_data["_id"]}, insert_data)
    # seed['reviews_status'] = 'success'
    # douban_movie_detail.updateOne({'_id': seed["_id"]}, seed)
    douban_movie_detail.close()
    douban_movie_reviews.close()
    return


def run_movie_reviews():
    print('run_movie_reviews')
    douban_movie_detail = mongo_manager('douban_movie_detail', db='douban')
    # douban_type_list.updateMany({}, {'movie_detail_status': '0'})
    # douban_type_list.close()
    while True:
        lists = []
        # for __ in range(900):
        #     seed = douban_type_list.find_one_and_update({'movie_detail_status':None},{'movie_detail_status':2})
        #     lists.append(seed)
        seeds = douban_movie_detail.findAll({'reviews_status': None}).limit(2)

        for seed in seeds:
            lists.append(seed)
        print('-' * 20, len(lists))
        pool = Pool(processes=min(1, len(lists) + 1))
        pool.map(save_movie_reviews, lists)
        pool.close()  # 关闭进程池，不再接受新的进程
        pool.join()
        if len(lists) < 100:
            break
        break


if __name__ == '__main__':
    run_movie_reviews()
